

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>geoslurp.dataset.CSVBase &mdash; geoslurp v1.1.0 documentation</title>
  

  
  
    <link rel="shortcut icon" href="../../../_static/favicon.ico"/>
  
  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  

  

  
        <link rel="index" title="Index"
              href="../../../genindex.html"/>
        <link rel="search" title="Search" href="../../../search.html"/>
    <link rel="top" title="geoslurp v1.1.0 documentation" href="../../../index.html"/>
        <link rel="up" title="Module code" href="../../index.html"/> 

  
  <script src="../../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="../../../index.html" class="icon icon-home"> geoslurp
          

          
          </a>

          
            
            
              <div class="version">
                1.1.0
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">Contents:</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../introduction.html">An introduction to Geoslurp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation of the geoslurp package and setting up the PostgreSQL instance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../confsettings.html">Configuration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../examples.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../reference.html">Reference Documentation</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../index.html">geoslurp</a>
        
      </nav>


      
      <div class="wy-nav-content">
        <div class="rst-content">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../../index.html">Module code</a> &raquo;</li>
        
      <li>geoslurp.dataset.CSVBase</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for geoslurp.dataset.CSVBase</h1><div class="highlight"><pre>
<span></span><span class="c1"># This file is part of geoslurp.</span>
<span class="c1"># geoslurp is free software; you can redistribute it and/or</span>
<span class="c1"># modify it under the terms of the GNU Lesser General Public</span>
<span class="c1"># License as published by the Free Software Foundation; either</span>
<span class="c1"># version 3 of the License, or (at your option) any later version.</span>

<span class="c1"># geoslurp is distributed in the hope that it will be useful,</span>
<span class="c1"># but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
<span class="c1"># MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU</span>
<span class="c1"># Lesser General Public License for more details.</span>

<span class="c1"># You should have received a copy of the GNU Lesser General Public</span>
<span class="c1"># License along with Frommle; if not, write to the Free Software</span>
<span class="c1"># Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA</span>

<span class="c1"># Author Roelof Rietbroek (roelof@geod.uni-bonn.de), 2018</span>

<span class="kn">from</span> <span class="nn">geoslurp.dataset.dataSetBase</span> <span class="k">import</span> <span class="n">DataSet</span>
<span class="kn">from</span> <span class="nn">geoslurp.config.slurplogger</span> <span class="k">import</span> <span class="n">slurplogger</span>
<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="n">Table</span><span class="p">,</span><span class="n">Column</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">String</span><span class="p">,</span> <span class="n">Float</span>
<span class="kn">from</span> <span class="nn">geoslurp.db</span> <span class="k">import</span> <span class="n">tableMapFactory</span>
<span class="kn">import</span> <span class="nn">re</span>

<div class="viewcode-block" id="columnsFromCSV"><a class="viewcode-back" href="../../../reference/geoslurp.dataset.html#geoslurp.dataset.CSVBase.columnsFromCSV">[docs]</a><span class="k">def</span> <span class="nf">columnsFromCSV</span><span class="p">(</span><span class="n">line</span><span class="p">,</span> <span class="n">lookup</span><span class="p">,</span><span class="n">sep</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;reads column descriptors from comma separated values and creates a list of sqlalchemy columns&quot;&quot;&quot;</span>
    <span class="n">names</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="n">cols</span> <span class="o">=</span> <span class="p">[</span><span class="n">Column</span><span class="p">(</span><span class="s1">&#39;id&#39;</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)]</span>
    <span class="n">ln</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">sep</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">ln</span><span class="p">:</span>
        <span class="n">ky</span> <span class="o">=</span> <span class="n">key</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
        <span class="n">names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ky</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
        <span class="n">cols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">Column</span><span class="p">(</span><span class="n">ky</span><span class="o">.</span><span class="n">lower</span><span class="p">(),</span> <span class="n">lookup</span><span class="p">[</span><span class="n">ky</span><span class="p">]))</span>
    <span class="k">return</span> <span class="n">names</span><span class="p">,</span> <span class="n">cols</span></div>


<div class="viewcode-block" id="valuesFromCSV"><a class="viewcode-back" href="../../../reference/geoslurp.dataset.html#geoslurp.dataset.CSVBase.valuesFromCSV">[docs]</a><span class="k">def</span> <span class="nf">valuesFromCSV</span><span class="p">(</span><span class="n">line</span><span class="p">,</span><span class="n">names</span><span class="p">,</span><span class="n">sep</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">):</span>
    <span class="n">vals</span><span class="o">=</span><span class="p">{}</span>
    <span class="k">for</span> <span class="n">ky</span><span class="p">,</span><span class="n">x</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span><span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">sep</span><span class="p">)):</span>
        <span class="n">val</span><span class="o">=</span><span class="n">x</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">bool</span><span class="p">(</span><span class="n">val</span><span class="p">):</span>
            <span class="k">continue</span>
        <span class="n">vals</span><span class="p">[</span><span class="n">ky</span><span class="p">]</span> <span class="o">=</span> <span class="n">val</span>
    <span class="k">return</span> <span class="n">vals</span></div>

<div class="viewcode-block" id="CSVBase"><a class="viewcode-back" href="../../../reference/geoslurp.dataset.html#geoslurp.dataset.CSVBase.CSVBase">[docs]</a><span class="k">class</span> <span class="nc">CSVBase</span><span class="p">(</span><span class="n">DataSet</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Base class which downloads reads in a CSV table and registers it in a db table&quot;&quot;&quot;</span>
    <span class="n">table</span><span class="o">=</span><span class="kc">None</span>
    <span class="n">csvfile</span><span class="o">=</span><span class="kc">None</span>
    <span class="n">separator</span><span class="o">=</span><span class="s1">&#39;,&#39;</span>
    <span class="n">lookup</span><span class="o">=</span><span class="kc">None</span>
    <span class="n">hskip</span><span class="o">=</span><span class="mi">0</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="n">dbconn</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dbconn</span><span class="p">)</span>
    
<div class="viewcode-block" id="CSVBase.register"><a class="viewcode-back" href="../../../reference/geoslurp.dataset.html#geoslurp.dataset.CSVBase.CSVBase.register">[docs]</a>    <span class="k">def</span> <span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Update/populate a database table  from a CSV file). This function reads all rows from an open CSV file. The first line is expected to hold the COlumn names, which are mapped to types in the lookup string dictionary</span>
<span class="sd">    &quot;&quot;&quot;</span>
        <span class="c1"># currently we can only cope with updating the entire table as a whole</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">dropTable</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span><span class="bp">self</span><span class="o">.</span><span class="n">scheme</span><span class="p">)</span>

        <span class="n">slurplogger</span><span class="p">()</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Filling CSV table </span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2"> with data from </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">csvfile</span><span class="p">))</span>
    
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">csvfile</span><span class="p">,</span><span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fid</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hskip</span><span class="p">):</span>
                <span class="nb">next</span><span class="p">(</span><span class="n">fid</span><span class="p">)</span>
            <span class="n">names</span><span class="p">,</span><span class="n">cols</span><span class="o">=</span><span class="n">columnsFromCSV</span><span class="p">(</span><span class="n">fid</span><span class="o">.</span><span class="n">readline</span><span class="p">(),</span><span class="bp">self</span><span class="o">.</span><span class="n">lookup</span><span class="p">)</span>
            <span class="n">table</span><span class="o">=</span><span class="n">Table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span><span class="bp">self</span><span class="o">.</span><span class="n">db</span><span class="o">.</span><span class="n">mdata</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">scheme</span><span class="p">)</span>
            <span class="n">table</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">checkfirst</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="o">=</span><span class="n">tableMapFactory</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>

            <span class="k">for</span> <span class="n">ln</span> <span class="ow">in</span> <span class="n">fid</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">addEntry</span><span class="p">(</span><span class="n">valuesFromCSV</span><span class="p">(</span><span class="n">ln</span><span class="p">,</span><span class="n">names</span><span class="p">))</span>


        <span class="c1">#also update entry in the inventory table</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">updateInvent</span><span class="p">()</span></div></div>

</pre></div>

           </div>
           <div class="articleComments">
            
           </div>
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2020, Roelof Rietbroek.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../../',
            VERSION:'v1.1.0',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: '.txt'
        };
    </script>
      <script type="text/javascript" src="../../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../../_static/doctools.js"></script>

  

  
  
    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
  

  
  
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>